package com.spark.mooc.ch5_rdd.part03_dataReadingAndWriting

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
import scala.util.parsing.json.JSON

/**
 * @description:
 * @time: 2020/11/27 14:36
 * @author: lhy
 */
object JSONRead {

    val inputFile: String = "input/rdd/people.json"

    def main(args: Array[String]): Unit = {
        val conf: SparkConf = new SparkConf().setAppName("JSONRead").setMaster("local")
        val sc = new SparkContext(conf)
        val jsonStrs: RDD[String] = sc.textFile(inputFile)
        val result: RDD[Option[Any]] = jsonStrs.map(s => JSON.parseFull(s))
        result.foreach({r => r match {
            case Some(map: Map[String,Any]) => println(map)
            case None => println("Parsing failed")
            case other => println("Unknown data.txt structure:" + other)
        }})
    }
}
